package com.xiezc.fetch.spider.biquge;

import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.DuplicateRemovedScheduler;

public class MySpider extends Spider {
    /**
     * create a spider with pageProcessor.
     *
     * @param pageProcessor pageProcessor
     */
    public MySpider(PageProcessor pageProcessor) {
        super(pageProcessor);
    }


    @Override
    public void close() {
        super.close();
        DuplicateRemovedScheduler scheduler = (DuplicateRemovedScheduler) this.scheduler;
       // scheduler.getDuplicateRemover().resetDuplicateCheck(null);
    }
}
